In [1]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import nltk
import string
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
In [2]:
data=pd.read_csv('../data/external/Shakespeare_data.csv')
Exploratory Data Analysis¶
In [3]:
data.head()
Out[3]:
In [4]:
data.info()
In [5]:
data["Player"].fillna("Other", inplace = True)
data.shape
Out[5]:
In [6]:
data=data.dropna()
data.reset_index(drop=True, inplace=True)
data.shape
Out[6]:
Number of lines each player has in a play¶
In [7]:
data.groupby(['Play', 'Player']).count()['Dataline']
Out[7]:
Number of players in each play¶
In [8]:
numberofPlayers = data.groupby(['Play'])['Player'].nunique().sort_values(ascending= False).to_frame()
numberofPlayers['Play'] = numberofPlayers.index.tolist()
numberofPlayers.columns = ['Number of Players','Play']
numberofPlayers.index= np.arange(0,len(numberofPlayers))
numberofPlayers
Out[8]:
No of players in each play - Plot¶
In [9]:
plt.figure(figsize=(8,8))
ax = sns.barplot(x='Number of Players',y='Play',data=numberofPlayers)
ax.set(xlabel='Number of Players', ylabel='Shakespeare Play')
plt.show()
# of dialogues per each player¶
In [10]:
numberoflines = data.groupby(['Player'])['PlayerLine'].nunique().sort_values(ascending= False).to_frame()
numberoflines['Player'] = numberoflines.index.tolist()
numberoflines.columns = ['Number of lines','Player']
numberoflines.index= np.arange(0,len(numberoflines))
numberoflines.sample(10)
Out[10]:
# of dialogues per each player-Plot¶
In [11]:
plt.figure(figsize=(10,15))
ax = sns.barplot(x='Number of lines',y='Player',data=numberoflines.head(45))
ax.set(xlabel='No of lines', ylabel='Player')
plt.show()
Feature Engineering¶
In [12]:
df1 = pd.DataFrame(data, columns = ['ActSceneLine','Play'])
df1.sample(5)
Out[12]:
Splitting the ActSceneLine in to three different columns¶
In [13]:
dataExpanded = data['ActSceneLine'].str.split('.', expand=True)
data['Act'] = dataExpanded[0]
data['Scene'] = dataExpanded[1]
data['SceneLine'] = dataExpanded[2]
data=data.drop(columns=['ActSceneLine'])
data.head()
Out[13]:
In [14]:
numberofActs=data.groupby(['Player']).max()['Act'].sort_values(ascending= False).to_frame()
numberofActs['Player'] = numberofActs.index.tolist()
numberofActs.columns = ['Number of Acts','Player']
numberofActs.index= np.arange(0,len(numberofActs))
numberofActs.sample(10)
data.dtypes
numberofActs['Number of Acts'] = numberofActs['Number of Acts'].astype(int)
Number of acts for each player¶
In [15]:
plt.figure(figsize=(10,15))
ax = sns.barplot(x='Number of Acts',y='Player',data=numberofActs.sample(100))
ax.set(xlabel='Number of Acts', ylabel='Player')
plt.show()
Word vectorization¶
Removing the punctuation marks in the player lines¶
In [16]:
data['PlayerLine'] = data['PlayerLine'].str.lower()
punctuation_marks = list("?:!.,;")
for punct_mark in punctuation_marks:
data['PlayerLine'] = data['PlayerLine'].str.replace(punct_mark, '')
In [17]:
nltk.download('punkt')
print("------------------------------------------------------------")
nltk.download('wordnet')
print("------------------------------------------------------------")
nltk.download('stopwords')
Out[17]:
Lemmatization of words¶
In [18]:
wordnet_lemmatizer = WordNetLemmatizer()
numberofplayerLines = len(data)
lemmatized_text_list = []
for row in range(0, numberofplayerLines):
lemmatized_word_list = []
dialogue = data.loc[row]['PlayerLine']
dialogue_words = dialogue.split(" ")
for word in dialogue_words:
lemmatized_word_list.append(wordnet_lemmatizer.lemmatize(word, pos="v"))
lemmatized_text = " ".join(lemmatized_word_list)
lemmatized_text_list.append(lemmatized_text)
In [19]:
data['PlayerLine'] = lemmatized_text_list
data.head()
Out[19]:
In [20]:
stop_words = list(stopwords.words('english'))
for stop_word in stop_words:
regex_stopword = r"\b" + stop_word + r"\b"
data['PlayerLine'] = data['PlayerLine'].str.replace(regex_stopword, '')
data['Word_tokens']=data['PlayerLine'].apply(lambda x:' '.join(w for w in nltk.word_tokenize(x.lower().strip()) if not w in stop_word) )
data.Word_tokens.head()
Out[20]:
Word Tokenization¶
In [21]:
data_play_content=pd.DataFrame(data.groupby('Player')['Word_tokens'].apply(lambda x: "{%s}" % ', '.join(x)))
data_play_content.head(10)
Out[21]:
Lets plot the wordcloud for sample of 20 players. Plotting them for the full player list take longer execution time¶
In [22]:
from wordcloud import WordCloud
def WordCloudGeneration(str1,title):
plt.figure(figsize=(20, 10))
wordcloudoflines = WordCloud( background_color='black',width=600, height=400, max_font_size=50).generate(str1)
wordcloudoflines.recolor(random_state=0)
plt.title(title, fontsize=60,color='red')
plt.imshow(wordcloudoflines)
plt.axis('off')
plt.show()
for cols in data_play_content.sample(20).index:
WordCloudGeneration(data_play_content.loc[cols,'Word_tokens'],cols)
TF-IDF vectorization¶
Considering only the top 100 features across the corpus
In [23]:
ngram_range = (1,2)
max_features = 100
min_df = 10
max_df = 20
tfidf = TfidfVectorizer(encoding='utf-8',
ngram_range=ngram_range,
lowercase=True,
norm='l2',
max_features=max_features,
max_df=max_df,
min_df=min_df,
sublinear_tf=True)
featuers_labels = tfidf.fit_transform(data['PlayerLine']).toarray()
Creating A data frame to fit into the model¶
In [24]:
f=pd.DataFrame(data=featuers_labels[0:,0:], index=[i for i in range(featuers_labels.shape[0])],columns=['f'+str(i) for i in range(featuers_labels.shape[1])])
f['Act'] = data['Act']
f['Scene'] = data['Scene']
f['SceneLine'] = data['SceneLine']
f['Play']=data['Play']
f = pd.get_dummies(f, columns=['Play'])
dftarget = pd.DataFrame(data, columns = [ 'Player'])
f.sample(10)
Out[24]:
Classification Using Various models¶
In [25]:
from sklearn.model_selection import train_test_split
xtrain, xtest, ytrain, ytest = train_test_split(f, dftarget, test_size=0.2)
In [26]:
print(xtrain.shape, xtest.shape)
In [27]:
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
model = GaussianNB()
dc = DecisionTreeClassifier()
rf = RandomForestClassifier()
model.fit(xtrain, ytrain)
dc.fit(xtrain, ytrain)
rf.fit(xtrain, ytrain)
Out[27]:
Accuracy Score with Naive Bayes, Decision Tree, Random Forest¶
In [28]:
from sklearn.metrics import accuracy_score
print('Accuracy for GaussianNB: ', 100 * accuracy_score( ytest, model.predict(xtest)), '%')
print('Accuracy for Decision Tree: ', 100 * accuracy_score(ytest, dc.predict(xtest)), '%')
print('Accuracy for random forest: ', 100 * accuracy_score(ytest, rf.predict(xtest)), '%')
Classification using Nearest Neighbour Classifier¶
In [29]:
from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier()
knn.fit(xtrain, ytrain)
print('Accuracy for nearest neighbour: ',100*accuracy_score(ytest, knn.predict(xtest)),'%')
Let's see how the accuracy varies depending upon the max_depth of a decision tree¶
In [30]:
dc1 = DecisionTreeClassifier(max_depth=10)
dc2 = DecisionTreeClassifier(max_depth=25)
dc3 = DecisionTreeClassifier(max_depth=50)
dc4 = DecisionTreeClassifier(max_depth=100)
dc5 = DecisionTreeClassifier(max_depth=150)
dc1.fit(xtrain , ytrain)
dc2.fit(xtrain , ytrain)
dc3.fit(xtrain , ytrain)
dc4.fit(xtrain , ytrain)
dc5.fit(xtrain , ytrain)
print('Max Depth of 10 :', 100*accuracy_score(ytest, dc1.predict(xtest)), '%')
print(accuracy_score(ytest, dc2.predict(xtest)))
print(accuracy_score(ytest, dc3.predict(xtest)))
print(accuracy_score(ytest, dc4.predict(xtest)))
print(accuracy_score(ytest, dc5.predict(xtest)))
Notes¶
- Performed EDA on the dataset
- Splitted the ActSceneLine Feature into 3 seperate columns to fit the model
- Coverted the PlayerLine into different Vectors using TFIDF vectorization
- Used Play, Act ,Scene, SceneLine, Player line columns to classify the players.
observations¶
- Accuracy is more with the Decision Tree and Random Forest Classification Models
- The accuracy in for this data set is varying with the maxdif and mindif of the TFIDF vectorization.
- Accuracy is also increasing with increase in the max_depth parameter of the DecisionTree Classifier and becomes constant after a certain depth